<!doctype html>



  


<html class="theme-next pisces use-motion" lang="zh-Hans">
<head>
  <meta charset="UTF-8"/>
<meta http-equiv="X-UA-Compatible" content="IE=edge" />
<meta name="viewport" content="width=device-width, initial-scale=1, maximum-scale=1"/>









<meta http-equiv="Cache-Control" content="no-transform" />
<meta http-equiv="Cache-Control" content="no-siteapp" />















  
  
  <link href="/lib/fancybox/source/jquery.fancybox.css?v=2.1.5" rel="stylesheet" type="text/css" />




  
  
  
  

  
    
    
  

  

  

  

  

  
    
    
    <link href="//fonts.googleapis.com/css?family=Lato:300,300italic,400,400italic,700,700italic&subset=latin,latin-ext" rel="stylesheet" type="text/css">
  






<link href="/lib/font-awesome/css/font-awesome.min.css?v=4.6.2" rel="stylesheet" type="text/css" />

<link href="/css/main.css?v=5.1.1" rel="stylesheet" type="text/css" />


  <meta name="keywords" content="java,kafka," />








  <link rel="shortcut icon" type="image/x-icon" href="/favicon.ico?v=5.1.1" />






<meta name="description" content="前言在上一篇中讲述如何搭建kafka集群，本篇则讲述如何简单的使用 kafka 。不过在使用kafka的时候，还是应该简单的了解下kafka。 Kafka的介绍Kafka是一种高吞吐量的分布式发布订阅消息系统，它可以处理消费者规模的网站中的所有动作流数据。Kafka 有如下特性：  以时间复杂度为O(1)的方式提供消息持久化能力，即使对TB级以上数据也能保证常数时间复杂度的访问性能。 高吞吐率。">
<meta name="keywords" content="java,kafka">
<meta property="og:type" content="article">
<meta property="og:title" content="Kafka 使用Java实现数据的生产和消费demo">
<meta property="og:url" content="http://yoursite.com/2018/01/28/pancm71/index.html">
<meta property="og:site_name" content="虚无境的博客">
<meta property="og:description" content="前言在上一篇中讲述如何搭建kafka集群，本篇则讲述如何简单的使用 kafka 。不过在使用kafka的时候，还是应该简单的了解下kafka。 Kafka的介绍Kafka是一种高吞吐量的分布式发布订阅消息系统，它可以处理消费者规模的网站中的所有动作流数据。Kafka 有如下特性：  以时间复杂度为O(1)的方式提供消息持久化能力，即使对TB级以上数据也能保证常数时间复杂度的访问性能。 高吞吐率。">
<meta property="og:image" content="http://img.blog.csdn.net/20180128125655810?watermark/2/text/aHR0cDovL2Jsb2cuY3Nkbi5uZXQvcWF6d3N4cGNt/font/5a6L5L2T/fontsize/400/fill/I0JBQkFCMA==/dissolve/70/gravity/SouthEast">
<meta property="og:image" content="http://img.blog.csdn.net/20180128140154683?watermark/2/text/aHR0cDovL2Jsb2cuY3Nkbi5uZXQvcWF6d3N4cGNt/font/5a6L5L2T/fontsize/400/fill/I0JBQkFCMA==/dissolve/70/gravity/SouthEast">
<meta property="og:image" content="http://img.blog.csdn.net/20180128142903484?watermark/2/text/aHR0cDovL2Jsb2cuY3Nkbi5uZXQvcWF6d3N4cGNt/font/5a6L5L2T/fontsize/400/fill/I0JBQkFCMA==/dissolve/70/gravity/SouthEast">
<meta property="og:updated_time" content="2018-02-09T11:56:38.000Z">
<meta name="twitter:card" content="summary">
<meta name="twitter:title" content="Kafka 使用Java实现数据的生产和消费demo">
<meta name="twitter:description" content="前言在上一篇中讲述如何搭建kafka集群，本篇则讲述如何简单的使用 kafka 。不过在使用kafka的时候，还是应该简单的了解下kafka。 Kafka的介绍Kafka是一种高吞吐量的分布式发布订阅消息系统，它可以处理消费者规模的网站中的所有动作流数据。Kafka 有如下特性：  以时间复杂度为O(1)的方式提供消息持久化能力，即使对TB级以上数据也能保证常数时间复杂度的访问性能。 高吞吐率。">
<meta name="twitter:image" content="http://img.blog.csdn.net/20180128125655810?watermark/2/text/aHR0cDovL2Jsb2cuY3Nkbi5uZXQvcWF6d3N4cGNt/font/5a6L5L2T/fontsize/400/fill/I0JBQkFCMA==/dissolve/70/gravity/SouthEast">



<script type="text/javascript" id="hexo.configurations">
  var NexT = window.NexT || {};
  var CONFIG = {
    root: '/',
    scheme: 'Pisces',
    sidebar: {"position":"left","display":"post","offset":12,"offset_float":0,"b2t":false,"scrollpercent":false},
    fancybox: true,
    motion: true,
    duoshuo: {
      userId: '0',
      author: '博主'
    },
    algolia: {
      applicationID: '',
      apiKey: '',
      indexName: '',
      hits: {"per_page":10},
      labels: {"input_placeholder":"Search for Posts","hits_empty":"We didn't find any results for the search: ${query}","hits_stats":"${hits} results found in ${time} ms"}
    }
  };
</script>



  <link rel="canonical" href="http://yoursite.com/2018/01/28/pancm71/"/>






  <title>Kafka 使用Java实现数据的生产和消费demo | 虚无境的博客</title>
  





  <script type="text/javascript">
    var _hmt = _hmt || [];
    (function() {
      var hm = document.createElement("script");
      hm.src = "https://hm.baidu.com/hm.js?39c177d10f6e05ddfa113e02139b9c1c";
      var s = document.getElementsByTagName("script")[0];
      s.parentNode.insertBefore(hm, s);
    })();
  </script>










</head>

<body itemscope itemtype="http://schema.org/WebPage" lang="zh-Hans">

  
  
    
  

  <div class="container sidebar-position-left page-post-detail ">
    <div class="headband"></div>

    <header id="header" class="header" itemscope itemtype="http://schema.org/WPHeader">
      <div class="header-inner"><div class="site-brand-wrapper">
  <div class="site-meta ">
    

    <div class="custom-logo-site-title">
      <a href="/"  class="brand" rel="start">
        <span class="logo-line-before"><i></i></span>
        <span class="site-title">虚无境的博客</span>
        <span class="logo-line-after"><i></i></span>
      </a>
    </div>
      
        <p class="site-subtitle"></p>
      
  </div>

  <div class="site-nav-toggle">
    <button>
      <span class="btn-bar"></span>
      <span class="btn-bar"></span>
      <span class="btn-bar"></span>
    </button>
  </div>
</div>

<nav class="site-nav">
  

  
    <ul id="menu" class="menu">
      
        
        <li class="menu-item menu-item-home">
          <a href="/" rel="section">
            
              <i class="menu-item-icon fa fa-fw fa-home"></i> <br />
            
            首页
          </a>
        </li>
      
        
        <li class="menu-item menu-item-categories">
          <a href="/categories/" rel="section">
            
              <i class="menu-item-icon fa fa-fw fa-th"></i> <br />
            
            分类
          </a>
        </li>
      
        
        <li class="menu-item menu-item-archives">
          <a href="/archives/" rel="section">
            
              <i class="menu-item-icon fa fa-fw fa-archive"></i> <br />
            
            归档
          </a>
        </li>
      
        
        <li class="menu-item menu-item-tags">
          <a href="/tags/" rel="section">
            
              <i class="menu-item-icon fa fa-fw fa-tags"></i> <br />
            
            标签
          </a>
        </li>
      
        
        <li class="menu-item menu-item-about">
          <a href="/about/" rel="section">
            
              <i class="menu-item-icon fa fa-fw fa-user"></i> <br />
            
            关于
          </a>
        </li>
      

      
    </ul>
  

  
</nav>



 </div>
    </header>

    <main id="main" class="main">
      <div class="main-inner">
        <div class="content-wrap">
          <div id="content" class="content">
            

  <div id="posts" class="posts-expand">
    

  

  
  
  

  <article class="post post-type-normal " itemscope itemtype="http://schema.org/Article">
    <link itemprop="mainEntityOfPage" href="http://yoursite.com/2018/01/28/pancm71/">

    <span hidden itemprop="author" itemscope itemtype="http://schema.org/Person">
      <meta itemprop="name" content="虚无境">
      <meta itemprop="description" content="">
      <meta itemprop="image" content="/images/xuwujing.png">
    </span>

    <span hidden itemprop="publisher" itemscope itemtype="http://schema.org/Organization">
      <meta itemprop="name" content="虚无境的博客">
    </span>

    
      <header class="post-header">

        
        
          <h1 class="post-title" itemprop="name headline">Kafka 使用Java实现数据的生产和消费demo</h1>
        

        <div class="post-meta">
          <span class="post-time">
            
              <span class="post-meta-item-icon">
                <i class="fa fa-calendar-o"></i>
              </span>
              
                <span class="post-meta-item-text">发表于</span>
              
              <time title="创建于" itemprop="dateCreated datePublished" datetime="2018-01-28T14:38:00+08:00">
                2018-01-28
              </time>
            

            

            
          </span>

          
            <span class="post-category" >
            
              <span class="post-meta-divider">|</span>
            
              <span class="post-meta-item-icon">
                <i class="fa fa-folder-o"></i>
              </span>
              
                <span class="post-meta-item-text">分类于</span>
              
              
                <span itemprop="about" itemscope itemtype="http://schema.org/Thing">
                  <a href="/categories/kafka/" itemprop="url" rel="index">
                    <span itemprop="name">kafka</span>
                  </a>
                </span>

                
                
              
            </span>
          

          
            
          

          
          

          

          

          

        </div>
      </header>
    

    <div class="post-body" itemprop="articleBody">

      
      

      
        <script src="\assets\js\APlayer.min.js"> </script><h2 id="前言"><a href="#前言" class="headerlink" title="前言"></a>前言</h2><p>在<a href="http://www.cnblogs.com/xuwujing/p/8361629.html" target="_blank" rel="external">上一篇</a>中讲述如何搭建kafka集群，本篇则讲述如何简单的使用 kafka 。不过在使用kafka的时候，还是应该简单的了解下kafka。</p>
<h2 id="Kafka的介绍"><a href="#Kafka的介绍" class="headerlink" title="Kafka的介绍"></a>Kafka的介绍</h2><p>Kafka是一种高吞吐量的分布式发布订阅消息系统，它可以处理消费者规模的网站中的所有动作流数据。<br>Kafka 有如下特性：</p>
<ul>
<li>以时间复杂度为O(1)的方式提供消息持久化能力，即使对TB级以上数据也能保证常数时间复杂度的访问性能。</li>
<li>高吞吐率。即使在非常廉价的商用机器上也能做到单机支持每秒100K条以上消息的传输。</li>
<li>支持Kafka Server间的消息分区，及分布式消费，同时保证每个Partition内的消息顺序传输。</li>
<li>同时支持离线数据处理和实时数据处理。</li>
<li>Scale out：支持在线水平扩展。</li>
</ul>
<h3 id="kafka的术语"><a href="#kafka的术语" class="headerlink" title="kafka的术语"></a>kafka的术语</h3><ul>
<li>Broker：Kafka集群包含一个或多个服务器，这种服务器被称为broker。</li>
<li>Topic：每条发布到Kafka集群的消息都有一个类别，这个类别被称为Topic。（物理上不同Topic的消息分开存储，逻辑上一个Topic的消息虽然保存于一个或多个broker上但用户只需指定消息的Topic即可生产或消费数据而不必关心数据存于何处）</li>
<li>Partition：Partition是物理上的概念，每个Topic包含一个或多个Partition。</li>
<li>Producer：负责发布消息到Kafka broker。</li>
<li>Consumer：消息消费者，向Kafka broker读取消息的客户端。</li>
<li>Consumer Group:每个Consumer属于一个特定的Consumer Group（可为每个Consumer指定group name，若不指定group name则属于默认的group）。</li>
</ul>
<h3 id="kafka核心Api"><a href="#kafka核心Api" class="headerlink" title="kafka核心Api"></a>kafka核心Api</h3><p>kafka有四个核心API</p>
<ul>
<li>应用程序使用producer API发布消息到1个或多个topic中。</li>
<li>应用程序使用consumer API来订阅一个或多个topic,并处理产生的消息。</li>
<li>应用程序使用streams API充当一个流处理器,从1个或多个topic消费输入流,并产生一个输出流到1个或多个topic,有效地将输入流转换到输出流。</li>
<li><p>connector API允许构建或运行可重复使用的生产者或消费者,将topic链接到现有的应用程序或数据系统。</p>
<p>示例图如下:<br><img src="http://img.blog.csdn.net/20180128125655810?watermark/2/text/aHR0cDovL2Jsb2cuY3Nkbi5uZXQvcWF6d3N4cGNt/font/5a6L5L2T/fontsize/400/fill/I0JBQkFCMA==/dissolve/70/gravity/SouthEast" alt="这里写图片描述"></p>
</li>
</ul>
<h3 id="kafka-应用场景"><a href="#kafka-应用场景" class="headerlink" title="kafka 应用场景"></a>kafka 应用场景</h3><ul>
<li>构建可在系统或应用程序之间可靠获取数据的实时流数据管道。</li>
<li>构建实时流应用程序，可以转换或响应数据流。</li>
</ul>
<p>以上介绍参考kafka官方文档。</p>
<h2 id="开发准备"><a href="#开发准备" class="headerlink" title="开发准备"></a>开发准备</h2><p>如果我们要开发一个kafka的程序，应该做些什么呢？<br>首先，在搭建好kafka环境之后，我们要考虑的是我们是生产者还是消费者，也就是消息的发送者还是接受者。<br>不过在本篇中，生产者和消费者都会进行开发和讲解。</p>
<p>在大致的了解kafka之后，我们来开发第一个程序。<br>这里用的开发语言是Java，构建工具Maven。<br>Maven的依赖如下:</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><div class="line">1</div><div class="line">2</div><div class="line">3</div><div class="line">4</div><div class="line">5</div><div class="line">6</div><div class="line">7</div><div class="line">8</div><div class="line">9</div><div class="line">10</div><div class="line">11</div><div class="line">12</div><div class="line">13</div><div class="line">14</div><div class="line">15</div><div class="line">16</div><div class="line">17</div><div class="line">18</div></pre></td><td class="code"><pre><div class="line">&lt;dependency&gt;</div><div class="line">  	 	&lt;groupId&gt;org.apache.kafka&lt;/groupId&gt;</div><div class="line">  		 &lt;artifactId&gt;kafka_2.12&lt;/artifactId&gt;</div><div class="line">  		 &lt;version&gt;1.0.0&lt;/version&gt;</div><div class="line">		&lt;scope&gt;provided&lt;/scope&gt; </div><div class="line">	&lt;/dependency&gt;</div><div class="line">	</div><div class="line">	&lt;dependency&gt;</div><div class="line">   		 &lt;groupId&gt;org.apache.kafka&lt;/groupId&gt;</div><div class="line">   		 &lt;artifactId&gt;kafka-clients&lt;/artifactId&gt;</div><div class="line">  		  &lt;version&gt;1.0.0&lt;/version&gt;</div><div class="line">	&lt;/dependency&gt;</div><div class="line">	</div><div class="line">	&lt;dependency&gt;</div><div class="line">	    &lt;groupId&gt;org.apache.kafka&lt;/groupId&gt;</div><div class="line">	    &lt;artifactId&gt;kafka-streams&lt;/artifactId&gt;</div><div class="line">	    &lt;version&gt;1.0.0&lt;/version&gt;</div><div class="line">	&lt;/dependency&gt;</div></pre></td></tr></table></figure>
<h2 id="Kafka-Producer"><a href="#Kafka-Producer" class="headerlink" title="Kafka Producer"></a>Kafka Producer</h2><p>在开发生产的时候，先简单的介绍下kafka各种配置说明：</p>
<ul>
<li>bootstrap.servers： kafka的地址。</li>
<li>acks:消息的确认机制，默认值是0。<br> acks=0：如果设置为0，生产者不会等待kafka的响应。<br> acks=1：这个配置意味着kafka会把这条消息写到本地日志文件中，但是不会等待集群中其他机器的成功响应。<br> acks=all：这个配置意味着leader会等待所有的follower同步完成。这个确保消息不会丢失，除非kafka集群中所有机器挂掉。这是最强的可用性保证。</li>
<li>retries：配置为大于0的值的话，客户端会在消息发送失败时重新发送。</li>
<li>batch.size:当多条消息需要发送到同一个分区时，生产者会尝试合并网络请求。这会提高client和生产者的效率。</li>
<li>key.serializer: 键序列化，默认org.apache.kafka.common.serialization.StringDeserializer。</li>
<li>value.deserializer:值序列化，默认org.apache.kafka.common.serialization.StringDeserializer。<br>…<br>还有更多配置，可以去查看官方文档，这里就不在说明了。<br>那么我们kafka 的producer配置如下:</li>
</ul>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><div class="line">1</div><div class="line">2</div><div class="line">3</div><div class="line">4</div><div class="line">5</div><div class="line">6</div><div class="line">7</div><div class="line">8</div></pre></td><td class="code"><pre><div class="line">Properties props = new Properties();</div><div class="line">props.put(&quot;bootstrap.servers&quot;, &quot;master:9092,slave1:9092,slave2:9092&quot;);</div><div class="line">props.put(&quot;acks&quot;, &quot;all&quot;);</div><div class="line">props.put(&quot;retries&quot;, 0);</div><div class="line">props.put(&quot;batch.size&quot;, 16384);</div><div class="line">props.put(&quot;key.serializer&quot;, StringSerializer.class.getName());</div><div class="line">props.put(&quot;value.serializer&quot;, StringSerializer.class.getName());</div><div class="line">KafkaProducer&lt;String, String&gt; producer = new KafkaProducer&lt;String, String&gt;(props);</div></pre></td></tr></table></figure>
<p> kafka的配置添加之后，我们便开始生产数据，生产数据代码只需如下就行:</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><div class="line">1</div></pre></td><td class="code"><pre><div class="line">producer.send(new ProducerRecord&lt;String, String&gt;(topic,key,value));</div></pre></td></tr></table></figure>
<ul>
<li>topic: 消息队列的名称，可以先行在kafka服务中进行创建。如果kafka中并未创建该topic，那么便会自动创建！</li>
<li>key:键值，也就是value对应的值，和Map类似。</li>
<li><p>value:要发送的数据，数据格式为String类型的。</p>
<p>在写好生产者程序之后，那我们先来生产吧！<br>我这里发送的消息为:</p>
</li>
</ul>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><div class="line">1</div></pre></td><td class="code"><pre><div class="line">String messageStr=&quot;你好，这是第&quot;+messageNo+&quot;条数据&quot;;</div></pre></td></tr></table></figure>
<p>并且只发送1000条就退出，结果如下:<br><img src="http://img.blog.csdn.net/20180128140154683?watermark/2/text/aHR0cDovL2Jsb2cuY3Nkbi5uZXQvcWF6d3N4cGNt/font/5a6L5L2T/fontsize/400/fill/I0JBQkFCMA==/dissolve/70/gravity/SouthEast" alt="这里写图片描述"></p>
<p>可以看到信息成功的打印了。<br>如果不想用程序进行验证程序是否发送成功，以及消息发送的准确性，可以在kafka服务器上使用命令查看。</p>
<h2 id="Kafka-Consumer"><a href="#Kafka-Consumer" class="headerlink" title="Kafka Consumer"></a>Kafka Consumer</h2><p>kafka消费这块应该来说是重点，毕竟大部分的时候，我们主要使用的是将数据进行消费。</p>
<p>kafka消费的配置如下:</p>
<ul>
<li>bootstrap.servers： kafka的地址。</li>
<li>group.id：组名 不同组名可以重复消费。例如你先使用了组名A消费了kafka的1000条数据，但是你还想再次进行消费这1000条数据，并且不想重新去产生，那么这里你只需要更改组名就可以重复消费了。</li>
<li>enable.auto.commit：是否自动提交，默认为true。</li>
<li>auto.commit.interval.ms: 从poll(拉)的回话处理时长。</li>
<li>session.timeout.ms:超时时间。</li>
<li>max.poll.records:一次最大拉取的条数。</li>
<li>auto.offset.reset：消费规则，默认earliest 。<br>earliest: 当各分区下有已提交的offset时，从提交的offset开始消费；无提交的offset时，从头开始消费 。<br>latest: 当各分区下有已提交的offset时，从提交的offset开始消费；无提交的offset时，消费新产生的该分区下的数据 。<br>none: topic各分区都存在已提交的offset时，从offset后开始消费；只要有一个分区不存在已提交的offset，则抛出异常。</li>
<li>key.serializer: 键序列化，默认org.apache.kafka.common.serialization.StringDeserializer。</li>
<li><p>value.deserializer:值序列化，默认org.apache.kafka.common.serialization.StringDeserializer。</p>
<p>那么我们kafka 的consumer配置如下:</p>
</li>
</ul>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><div class="line">1</div><div class="line">2</div><div class="line">3</div><div class="line">4</div><div class="line">5</div><div class="line">6</div><div class="line">7</div><div class="line">8</div><div class="line">9</div><div class="line">10</div><div class="line">11</div></pre></td><td class="code"><pre><div class="line">Properties props = new Properties();</div><div class="line">	props.put(&quot;bootstrap.servers&quot;, &quot;master:9092,slave1:9092,slave2:9092&quot;);</div><div class="line">	props.put(&quot;group.id&quot;, GROUPID);</div><div class="line">	props.put(&quot;enable.auto.commit&quot;, &quot;true&quot;);</div><div class="line">	props.put(&quot;auto.commit.interval.ms&quot;, &quot;1000&quot;);</div><div class="line">	props.put(&quot;session.timeout.ms&quot;, &quot;30000&quot;);</div><div class="line">	props.put(&quot;max.poll.records&quot;, 1000);</div><div class="line">	props.put(&quot;auto.offset.reset&quot;, &quot;earliest&quot;);</div><div class="line">	props.put(&quot;key.deserializer&quot;, StringDeserializer.class.getName());</div><div class="line">	props.put(&quot;value.deserializer&quot;, StringDeserializer.class.getName());</div><div class="line">	KafkaConsumer&lt;String, String&gt; consumer = new KafkaConsumer&lt;String, String&gt;(props);</div></pre></td></tr></table></figure>
<p>由于我这是设置的自动提交，所以消费代码如下:<br>我们需要先订阅一个topic，也就是指定消费哪一个topic。</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><div class="line">1</div></pre></td><td class="code"><pre><div class="line">consumer.subscribe(Arrays.asList(topic));</div></pre></td></tr></table></figure>
<p>订阅之后，我们再从kafka中拉取数据:</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><div class="line">1</div></pre></td><td class="code"><pre><div class="line">ConsumerRecords&lt;String, String&gt; msgList=consumer.poll(1000);</div></pre></td></tr></table></figure>
<p>一般来说进行消费会使用监听，这里我们就用for(;;)来进行监听， 并且设置消费1000条就退出！<br>结果如下:<br><img src="http://img.blog.csdn.net/20180128142903484?watermark/2/text/aHR0cDovL2Jsb2cuY3Nkbi5uZXQvcWF6d3N4cGNt/font/5a6L5L2T/fontsize/400/fill/I0JBQkFCMA==/dissolve/70/gravity/SouthEast" alt="这里写图片描述"></p>
<p>可以看到我们这里已经成功消费了生产的数据了。</p>
<h2 id="代码"><a href="#代码" class="headerlink" title="代码"></a>代码</h2><p>那么生产者和消费者的代码如下:</p>
<p><strong>生产者:</strong></p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><div class="line">1</div><div class="line">2</div><div class="line">3</div><div class="line">4</div><div class="line">5</div><div class="line">6</div><div class="line">7</div><div class="line">8</div><div class="line">9</div><div class="line">10</div><div class="line">11</div><div class="line">12</div><div class="line">13</div><div class="line">14</div><div class="line">15</div><div class="line">16</div><div class="line">17</div><div class="line">18</div><div class="line">19</div><div class="line">20</div><div class="line">21</div><div class="line">22</div><div class="line">23</div><div class="line">24</div><div class="line">25</div><div class="line">26</div><div class="line">27</div><div class="line">28</div><div class="line">29</div><div class="line">30</div><div class="line">31</div><div class="line">32</div><div class="line">33</div><div class="line">34</div><div class="line">35</div><div class="line">36</div><div class="line">37</div><div class="line">38</div><div class="line">39</div><div class="line">40</div><div class="line">41</div><div class="line">42</div><div class="line">43</div><div class="line">44</div><div class="line">45</div><div class="line">46</div><div class="line">47</div><div class="line">48</div><div class="line">49</div><div class="line">50</div><div class="line">51</div><div class="line">52</div><div class="line">53</div><div class="line">54</div><div class="line">55</div><div class="line">56</div><div class="line">57</div><div class="line">58</div><div class="line">59</div><div class="line">60</div><div class="line">61</div></pre></td><td class="code"><pre><div class="line">import java.util.Properties;</div><div class="line">import org.apache.kafka.clients.producer.KafkaProducer;</div><div class="line">import org.apache.kafka.clients.producer.ProducerRecord;</div><div class="line">import org.apache.kafka.common.serialization.StringSerializer;</div><div class="line"></div><div class="line">/**</div><div class="line"> * </div><div class="line">* Title: KafkaProducerTest</div><div class="line">* Description: </div><div class="line">* kafka 生产者demo</div><div class="line">* Version:1.0.0  </div><div class="line">* @author pancm</div><div class="line">* @date 2018年1月26日</div><div class="line"> */</div><div class="line">public class KafkaProducerTest implements Runnable &#123;</div><div class="line"></div><div class="line">	private final KafkaProducer&lt;String, String&gt; producer;</div><div class="line">	private final String topic;</div><div class="line">	public KafkaProducerTest(String topicName) &#123;</div><div class="line">		Properties props = new Properties();</div><div class="line">		props.put(&quot;bootstrap.servers&quot;, &quot;master:9092,slave1:9092,slave2:9092&quot;);</div><div class="line">		props.put(&quot;acks&quot;, &quot;all&quot;);</div><div class="line">		props.put(&quot;retries&quot;, 0);</div><div class="line">		props.put(&quot;batch.size&quot;, 16384);</div><div class="line">		props.put(&quot;key.serializer&quot;, StringSerializer.class.getName());</div><div class="line">		props.put(&quot;value.serializer&quot;, StringSerializer.class.getName());</div><div class="line">		this.producer = new KafkaProducer&lt;String, String&gt;(props);</div><div class="line">		this.topic = topicName;</div><div class="line">	&#125;</div><div class="line"></div><div class="line">	@Override</div><div class="line">	public void run() &#123;</div><div class="line">		int messageNo = 1;</div><div class="line">		try &#123;</div><div class="line">			for(;;) &#123;</div><div class="line">				String messageStr=&quot;你好，这是第&quot;+messageNo+&quot;条数据&quot;;</div><div class="line">				producer.send(new ProducerRecord&lt;String, String&gt;(topic, &quot;Message&quot;, messageStr));</div><div class="line">				//生产了100条就打印</div><div class="line">				if(messageNo%100==0)&#123;</div><div class="line">					System.out.println(&quot;发送的信息:&quot; + messageStr);</div><div class="line">				&#125;</div><div class="line">				//生产1000条就退出</div><div class="line">				if(messageNo%1000==0)&#123;</div><div class="line">					System.out.println(&quot;成功发送了&quot;+messageNo+&quot;条&quot;);</div><div class="line">					break;</div><div class="line">				&#125;</div><div class="line">				messageNo++;</div><div class="line">			&#125;</div><div class="line">		&#125; catch (Exception e) &#123;</div><div class="line">			e.printStackTrace();</div><div class="line">		&#125; finally &#123;</div><div class="line">			producer.close();</div><div class="line">		&#125;</div><div class="line">	&#125;</div><div class="line">	</div><div class="line">	public static void main(String args[]) &#123;</div><div class="line">		KafkaProducerTest test = new KafkaProducerTest(&quot;KAFKA_TEST&quot;);</div><div class="line">		Thread thread = new Thread(test);</div><div class="line">		thread.start();</div><div class="line">	&#125;</div><div class="line">&#125;</div></pre></td></tr></table></figure>
<p><strong>消费者:</strong></p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><div class="line">1</div><div class="line">2</div><div class="line">3</div><div class="line">4</div><div class="line">5</div><div class="line">6</div><div class="line">7</div><div class="line">8</div><div class="line">9</div><div class="line">10</div><div class="line">11</div><div class="line">12</div><div class="line">13</div><div class="line">14</div><div class="line">15</div><div class="line">16</div><div class="line">17</div><div class="line">18</div><div class="line">19</div><div class="line">20</div><div class="line">21</div><div class="line">22</div><div class="line">23</div><div class="line">24</div><div class="line">25</div><div class="line">26</div><div class="line">27</div><div class="line">28</div><div class="line">29</div><div class="line">30</div><div class="line">31</div><div class="line">32</div><div class="line">33</div><div class="line">34</div><div class="line">35</div><div class="line">36</div><div class="line">37</div><div class="line">38</div><div class="line">39</div><div class="line">40</div><div class="line">41</div><div class="line">42</div><div class="line">43</div><div class="line">44</div><div class="line">45</div><div class="line">46</div><div class="line">47</div><div class="line">48</div><div class="line">49</div><div class="line">50</div><div class="line">51</div><div class="line">52</div><div class="line">53</div><div class="line">54</div><div class="line">55</div><div class="line">56</div><div class="line">57</div><div class="line">58</div><div class="line">59</div><div class="line">60</div><div class="line">61</div><div class="line">62</div><div class="line">63</div><div class="line">64</div><div class="line">65</div><div class="line">66</div><div class="line">67</div><div class="line">68</div><div class="line">69</div><div class="line">70</div><div class="line">71</div><div class="line">72</div><div class="line">73</div><div class="line">74</div></pre></td><td class="code"><pre><div class="line">import java.util.Arrays;</div><div class="line">import java.util.Properties;</div><div class="line">import org.apache.kafka.clients.consumer.ConsumerRecord;</div><div class="line">import org.apache.kafka.clients.consumer.ConsumerRecords;</div><div class="line">import org.apache.kafka.clients.consumer.KafkaConsumer;</div><div class="line">import org.apache.kafka.common.serialization.StringDeserializer;</div><div class="line"></div><div class="line"></div><div class="line">/**</div><div class="line"> * </div><div class="line">* Title: KafkaConsumerTest</div><div class="line">* Description: </div><div class="line">*  kafka消费者 demo</div><div class="line">* Version:1.0.0  </div><div class="line">* @author pancm</div><div class="line">* @date 2018年1月26日</div><div class="line"> */</div><div class="line">public class KafkaConsumerTest implements Runnable &#123;</div><div class="line"></div><div class="line">	private final KafkaConsumer&lt;String, String&gt; consumer;</div><div class="line">	private ConsumerRecords&lt;String, String&gt; msgList;</div><div class="line">	private final String topic;</div><div class="line">	private static final String GROUPID = &quot;groupA&quot;;</div><div class="line"></div><div class="line">	public KafkaConsumerTest(String topicName) &#123;</div><div class="line">		Properties props = new Properties();</div><div class="line">		props.put(&quot;bootstrap.servers&quot;, &quot;master:9092,slave1:9092,slave2:9092&quot;);</div><div class="line">		props.put(&quot;group.id&quot;, GROUPID);</div><div class="line">		props.put(&quot;enable.auto.commit&quot;, &quot;true&quot;);</div><div class="line">		props.put(&quot;auto.commit.interval.ms&quot;, &quot;1000&quot;);</div><div class="line">		props.put(&quot;session.timeout.ms&quot;, &quot;30000&quot;);</div><div class="line">		props.put(&quot;auto.offset.reset&quot;, &quot;earliest&quot;);</div><div class="line">		props.put(&quot;key.deserializer&quot;, StringDeserializer.class.getName());</div><div class="line">		props.put(&quot;value.deserializer&quot;, StringDeserializer.class.getName());</div><div class="line">		this.consumer = new KafkaConsumer&lt;String, String&gt;(props);</div><div class="line">		this.topic = topicName;</div><div class="line">		this.consumer.subscribe(Arrays.asList(topic));</div><div class="line">	&#125;</div><div class="line"></div><div class="line">	@Override</div><div class="line">	public void run() &#123;</div><div class="line">		int messageNo = 1;</div><div class="line">		System.out.println(&quot;---------开始消费---------&quot;);</div><div class="line">		try &#123;</div><div class="line">			for (;;) &#123;</div><div class="line">					msgList = consumer.poll(1000);</div><div class="line">					if(null!=msgList&amp;&amp;msgList.count()&gt;0)&#123;</div><div class="line">					for (ConsumerRecord&lt;String, String&gt; record : msgList) &#123;</div><div class="line">						//消费100条就打印 ,但打印的数据不一定是这个规律的</div><div class="line">						if(messageNo%100==0)&#123;</div><div class="line">							System.out.println(messageNo+&quot;=======receive: key = &quot; + record.key() + &quot;, value = &quot; + record.value()+&quot; offset===&quot;+record.offset());</div><div class="line">						&#125;</div><div class="line">						//当消费了1000条就退出</div><div class="line">						if(messageNo%1000==0)&#123;</div><div class="line">							break;</div><div class="line">						&#125;</div><div class="line">						messageNo++;</div><div class="line">					&#125;</div><div class="line">				&#125;else&#123;	</div><div class="line">					Thread.sleep(1000);</div><div class="line">				&#125;</div><div class="line">			&#125;		</div><div class="line">		&#125; catch (InterruptedException e) &#123;</div><div class="line">			e.printStackTrace();</div><div class="line">		&#125; finally &#123;</div><div class="line">			consumer.close();</div><div class="line">		&#125;</div><div class="line">	&#125;  </div><div class="line">	public static void main(String args[]) &#123;</div><div class="line">		KafkaConsumerTest test1 = new KafkaConsumerTest(&quot;KAFKA_TEST&quot;);</div><div class="line">		Thread thread1 = new Thread(test1);</div><div class="line">		thread1.start();</div><div class="line">	&#125;</div><div class="line">&#125;</div></pre></td></tr></table></figure>
<p><strong>注:</strong> master、slave1、slave2 是因为我在自己的环境做了关系映射，这个可以换成服务器的IP。</p>
<p>当然项目我放在Github上了，有兴趣的可以看看。 <a href="https://github.com/xuwujing/kafka" target="_blank" rel="external">https://github.com/xuwujing/kafka</a></p>
<h2 id="总结"><a href="#总结" class="headerlink" title="总结"></a>总结</h2><p>简单的开发一个kafka的程序需要以下步骤:</p>
<ol>
<li>成功搭建kafka服务器，并成功启动！</li>
<li>得到kafka服务信息，然后在代码中进行相应的配置。</li>
<li>配置完成之后，监听kafka中的消息队列是否有消息产生。</li>
<li>将产生的数据进行业务逻辑处理！</li>
</ol>
<p>kafka介绍参考官方文档:<br><a href="http://kafka.apache.org/intro" target="_blank" rel="external">http://kafka.apache.org/intro</a></p>
<p>到此，本文就结束了，谢谢阅读！</p>

      
    </div>

    <div>
      
        

      
    </div>

    <div>
      
        

      
    </div>

    <div>
      
        

      
    </div>
     
    <div>
	 +
	  
<div style="text-align:center;color: #ccc;font-size:14px;">
------ 本文结束 ------</div>
<br/>
<div style="border: 1px solid black">
<div style="margin-left:10px">
<span style="font-weight:blod">版权声明</span>
<!-- <img src="/images/xuwujing.png" > -->
<br/>
<p style="font-size: 10px;line-height: 30px"><a href="http://www.panchengming.com/" style="color:#258FC6">xuwujing's Notes</a> by <a href="http://www.panchengming.com/" style="color:#258FC6">ChengMing Pan</a> is licensed under a <a href="https://creativecommons.org/licenses/by-nc-nd/4.0/" style="color:#258FC6">Creative Commons BY-NC-ND 4.0 International License</a>.<br/>
由<a href="http://www.panchengming.com/" style="color:#258FC6">虚无境</a>创作并维护的<a href="http://www.panchengming.com/" style="color:#258FC6">xuwujing's Notes</a>博客采用<a href="https://creativecommons.org/licenses/by-nc-nd/4.0/" style="color:#258FC6">创作共用保留署名-非商业-禁止演绎4.0国际许可证</a>。<br/>
本文首发于<a href="http://www.panchengming.com/" style="color:#258FC6">xuwujing's Notes</a> 博客（ <a href="http://www.panchengming.com/" style="color:#258FC6">http://www.panchengming.com/</a> ），版权所有，侵权必究。</p>
</div>
</div>

	
	</div>

    <footer class="post-footer">
      
        <div class="post-tags">
          
            <a href="/tags/java/" rel="tag"># java</a>
          
            <a href="/tags/kafka/" rel="tag"># kafka</a>
          
        </div>
      

      
      
      

      
        <div class="post-nav">
          <div class="post-nav-next post-nav-item">
            
              <a href="/2018/01/26/pancm70/" rel="next" title="kafka和storm集群的环境安装">
                <i class="fa fa-chevron-left"></i> kafka和storm集群的环境安装
              </a>
            
          </div>

          <span class="post-nav-divider"></span>

          <div class="post-nav-prev post-nav-item">
            
              <a href="/2018/02/08/pancm72/" rel="prev" title="关于Kafka 的 consumer 消费者手动提交详解">
                关于Kafka 的 consumer 消费者手动提交详解 <i class="fa fa-chevron-right"></i>
              </a>
            
          </div>
        </div>
      

      
      
    </footer>
  </article>



    <div class="post-spread">
      
    </div>
  </div>


          </div>
          


          
  <div class="comments" id="comments">
    
  </div>


        </div>
        
          
  
  <div class="sidebar-toggle">
    <div class="sidebar-toggle-line-wrap">
      <span class="sidebar-toggle-line sidebar-toggle-line-first"></span>
      <span class="sidebar-toggle-line sidebar-toggle-line-middle"></span>
      <span class="sidebar-toggle-line sidebar-toggle-line-last"></span>
    </div>
  </div>

  <aside id="sidebar" class="sidebar">
    <div class="sidebar-inner">

      

      
        <ul class="sidebar-nav motion-element">
          <li class="sidebar-nav-toc sidebar-nav-active" data-target="post-toc-wrap" >
            文章目录
          </li>
          <li class="sidebar-nav-overview" data-target="site-overview">
            站点概览
          </li>
        </ul>
      

      <section class="site-overview sidebar-panel">
        <div class="site-author motion-element" itemprop="author" itemscope itemtype="http://schema.org/Person">
          <img class="site-author-image" itemprop="image"
               src="/images/xuwujing.png"
               alt="虚无境" />
          <p class="site-author-name" itemprop="name">虚无境</p>
           
              <p class="site-description motion-element" itemprop="description">The way of the future!</p>
          
        </div>
        <nav class="site-state motion-element">

          
            <div class="site-state-item site-state-posts">
              <a href="/archives/">
                <span class="site-state-item-count">136</span>
                <span class="site-state-item-name">日志</span>
              </a>
            </div>
          

          
            
            
            <div class="site-state-item site-state-categories">
              <a href="/categories/index.html">
                <span class="site-state-item-count">30</span>
                <span class="site-state-item-name">分类</span>
              </a>
            </div>
          

          
            
            
            <div class="site-state-item site-state-tags">
              <a href="/tags/index.html">
                <span class="site-state-item-count">59</span>
                <span class="site-state-item-name">标签</span>
              </a>
            </div>
          

        </nav>

        

        <div class="links-of-author motion-element">
          
            
              <span class="links-of-author-item">
                <a href="https://github.com/xuwujing" target="_blank" title="github">
                  
                    <i class="fa fa-fw fa-globe"></i>
                  
                  github
                </a>
              </span>
            
              <span class="links-of-author-item">
                <a href="http://blog.csdn.net/qazwsxpcm?viewmode=list" target="_blank" title="csdn">
                  
                    <i class="fa fa-fw fa-globe"></i>
                  
                  csdn
                </a>
              </span>
            
              <span class="links-of-author-item">
                <a href="https://home.cnblogs.com/u/xuwujing/" target="_blank" title="cnblogs">
                  
                    <i class="fa fa-fw fa-globe"></i>
                  
                  cnblogs
                </a>
              </span>
            
          
        </div>

        
        

        
        
          <div class="links-of-blogroll motion-element links-of-blogroll-inline">
            <div class="links-of-blogroll-title">
              <i class="fa  fa-fw fa-globe"></i>
              
            </div>
            <ul class="links-of-blogroll-list">
              
                <li class="links-of-blogroll-item">
                  <a href="http://www.woainia.site/" title="woainia" target="_blank">woainia</a>
                </li>
              
                <li class="links-of-blogroll-item">
                  <a href="http://cmsblogs.com/" title="chenssy" target="_blank">chenssy</a>
                </li>
              
                <li class="links-of-blogroll-item">
                  <a href="http://italker.imisty.cn" title="xiaowu" target="_blank">xiaowu</a>
                </li>
              
            </ul>
          </div>
        

        


      </section>

      
      <!--noindex-->
        <section class="post-toc-wrap motion-element sidebar-panel sidebar-panel-active">
          <div class="post-toc">

            
              
            

            
              <div class="post-toc-content"><ol class="nav"><li class="nav-item nav-level-2"><a class="nav-link" href="#前言"><span class="nav-number">1.</span> <span class="nav-text">前言</span></a></li><li class="nav-item nav-level-2"><a class="nav-link" href="#Kafka的介绍"><span class="nav-number">2.</span> <span class="nav-text">Kafka的介绍</span></a><ol class="nav-child"><li class="nav-item nav-level-3"><a class="nav-link" href="#kafka的术语"><span class="nav-number">2.1.</span> <span class="nav-text">kafka的术语</span></a></li><li class="nav-item nav-level-3"><a class="nav-link" href="#kafka核心Api"><span class="nav-number">2.2.</span> <span class="nav-text">kafka核心Api</span></a></li><li class="nav-item nav-level-3"><a class="nav-link" href="#kafka-应用场景"><span class="nav-number">2.3.</span> <span class="nav-text">kafka 应用场景</span></a></li></ol></li><li class="nav-item nav-level-2"><a class="nav-link" href="#开发准备"><span class="nav-number">3.</span> <span class="nav-text">开发准备</span></a></li><li class="nav-item nav-level-2"><a class="nav-link" href="#Kafka-Producer"><span class="nav-number">4.</span> <span class="nav-text">Kafka Producer</span></a></li><li class="nav-item nav-level-2"><a class="nav-link" href="#Kafka-Consumer"><span class="nav-number">5.</span> <span class="nav-text">Kafka Consumer</span></a></li><li class="nav-item nav-level-2"><a class="nav-link" href="#代码"><span class="nav-number">6.</span> <span class="nav-text">代码</span></a></li><li class="nav-item nav-level-2"><a class="nav-link" href="#总结"><span class="nav-number">7.</span> <span class="nav-text">总结</span></a></li></ol></div>
            

          </div>
        </section>
      <!--/noindex-->
      

      

    </div>
  </aside>


        
      </div>
    </main>

    <footer id="footer" class="footer">
      <div class="footer-inner">
        <div class="copyright" >
  
  &copy; 
  <span itemprop="copyrightYear">2021</span>
  <span class="with-love">
    <i class="fa fa-heart"></i>
  </span>
  <span class="author" itemprop="copyrightHolder">虚无境</span>
</div>


<div class="powered-by">
  由 <a class="theme-link" href="https://hexo.io">Hexo</a> 强力驱动
</div>

<div class="theme-info">
  主题 -
  <a class="theme-link" href="https://github.com/iissnan/hexo-theme-next">
    NexT.Pisces
  </a>
</div>

  <script async src="//busuanzi.ibruce.info/busuanzi/2.3/busuanzi.pure.mini.js">
</script>
 | 
本站总访问量<span id="busuanzi_value_site_pv"></span>次
 | 
本站访客数<span id="busuanzi_value_site_uv"></span>人次
<iframe frameborder="no" border="0" marginwidth="0" marginheight="0" width=330 height=86 src="//music.163.com/outchain/player?type=2&id=857896&auto=0&height=66"></iframe>



        

        
      </div>
    </footer>

    
      <div class="back-to-top">
        <i class="fa fa-arrow-up"></i>
        
      </div>
    

  </div>

  

<script type="text/javascript">
  if (Object.prototype.toString.call(window.Promise) !== '[object Function]') {
    window.Promise = null;
  }
</script>









  












  
  <script type="text/javascript" src="/lib/jquery/index.js?v=2.1.3"></script>

  
  <script type="text/javascript" src="/lib/fastclick/lib/fastclick.min.js?v=1.0.6"></script>

  
  <script type="text/javascript" src="/lib/jquery_lazyload/jquery.lazyload.js?v=1.9.7"></script>

  
  <script type="text/javascript" src="/lib/velocity/velocity.min.js?v=1.2.1"></script>

  
  <script type="text/javascript" src="/lib/velocity/velocity.ui.min.js?v=1.2.1"></script>

  
  <script type="text/javascript" src="/lib/fancybox/source/jquery.fancybox.pack.js?v=2.1.5"></script>


  


  <script type="text/javascript" src="/js/src/utils.js?v=5.1.1"></script>

  <script type="text/javascript" src="/js/src/motion.js?v=5.1.1"></script>



  
  


  <script type="text/javascript" src="/js/src/affix.js?v=5.1.1"></script>

  <script type="text/javascript" src="/js/src/schemes/pisces.js?v=5.1.1"></script>



  
  <script type="text/javascript" src="/js/src/scrollspy.js?v=5.1.1"></script>
<script type="text/javascript" src="/js/src/post-details.js?v=5.1.1"></script>



  


  <script type="text/javascript" src="/js/src/bootstrap.js?v=5.1.1"></script>



  


  




	





  





  





  






  





  

  

  

  

  

  

</body>
</html>
